// SPDX-FileCopyrightText: 2023-2025 Sangfor Technologies Inc.
// SPDX-License-Identifier: Mulan PSL v2
package resource

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"code.geniusai.org/notebook-manager/pkg/config"
	"github.com/google/uuid"

	"code.geniusai.org/notebook-manager/pkg/handler/data_report"

	"code.geniusai.org/notebook-manager/pkg/entity"
	"code.geniusai.org/notebook-manager/pkg/log"
	"code.geniusai.org/notebook-manager/pkg/utils/lxcfs"
	v1 "github.com/kubeflow/kubeflow/components/notebook-controller/api/v1"
	"gorm.io/gorm"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
	v1ResourceGpu             = "virtaitech.com/gpu"
	nvidiaResourceGpu         = "nvidia.com/gpu"
	cpuUnit                   = 1000
	memoryUnit                = 1024
	schedulerName             = "orion-scheduler"
	schedulerRule             = "local-only"
	idleTime                  = "7200"
	MemoryShmName             = "dshm"
	MemoryShmMouthPath        = "/dev/shm"
	DevFuseName               = "devfuse"
	DevFuseNamePath           = "/dev/fuse"
	MntDataS3fs               = "mntdatas3fs"
	MntDataS3fsPath           = "/mnt/data-s3fs"
	MntDataS3fsContainerPath  = "/var/s3:shared"
	GpuNodeSelectorLabelKey   = "gputype"
	NodeSelectorLabelKey      = "nodetype"
	NodeSelectorLabelValue    = "cpu"
	GetTrainModelWorkspaceUrl = "/train-manage/api/v1/projects/workspace"
)

// 创建Notebook
func (m *Manager) CreateNotebook(notebook *entity.Notebook) (*entity.Notebook, error) {
	// TODO check if have enough resource
	var err error
	var pvc *entity.Pvc

	notebook.ID = genIDByProjectUidName(notebook.UserID, notebook.Name, notebook.ProjectID)
	_, err = m.notebookStore.GetNotebook(notebook.ID)
	if err == nil {
		return nil, entity.NewErrorf(entity.CodeParamInvalid, entity.MsgNotebookExisted, notebook.Name)
	}

	k8sNotebook := notebook2K8sNotebook(notebook, m.conf)
	newK8sPvc := getNewPvcByNotebook(notebook)
	// dry-run create pvc
	// eg: volume existed; no enough resource
	if newK8sPvc != nil {
		pvc = &entity.Pvc{
			ID:     genIDByUidName(notebook.UserID, notebook.VolumeName),
			Name:   notebook.VolumeName,
			UserID: notebook.UserID,
			Size:   notebook.WorkspaceSize,
		}
		_, err = m.pvcStore.GetPvc(pvc.ID)
		if err == nil {
			return nil, entity.NewErrorf(entity.CodeParamInvalid, entity.MsgVolumeExisted, notebook.VolumeName)
		}
		//尝试创建pvc并不是真的创建,通过client.DryRunAll参数实现
		dryErr := m.k8sControllerClient.Create(context.Background(), newK8sPvc, client.DryRunAll)
		if dryErr != nil {
			log.Errorf("create k8s pvc error: %v", dryErr)
			return nil, k8sErrToUI(dryErr)
		}
	}

	getPvc, err := m.pvcStore.GetUserPvcByName(notebook.VolumeName, notebook.UserID)
	log.Infof("myPvc为", getPvc)
	if newK8sPvc == nil {
		notebook.WorkspaceSize = getPvc.Size
	}

	//尝试创建k8sNotebook并不是真的创建,通过client.DryRunAll参数实现
	dryErr := m.k8sControllerClient.Create(context.Background(), k8sNotebook, client.DryRunAll)
	if dryErr != nil {
		log.Errorf("create k8s notebook error: %v", dryErr)
		return nil, k8sErrToUI(dryErr)
	}

	var retNotebook *entity.Notebook
	err = m.db.Transaction(func(tx *gorm.DB) error {
		if newK8sPvc != nil {
			log.Infof("creating k8s pvc(namespace: %v, name: %v)", newK8sPvc.Namespace, newK8sPvc.Name)
			cErr := m.k8sControllerClient.Create(context.Background(), newK8sPvc)
			if cErr != nil {
				log.Errorf("create k8s pvc(ID: %v) error: %v", pvc.ID, cErr)
				return k8sErrToUI(cErr)
			}
			// create pvc in db
			log.Infof("creating db pvc(ID: %v)", pvc.ID)
			_, cErr = m.pvcStore.CreatePvc(tx, pvc)
			if cErr != nil {
				log.Errorf("create db pvc(ID: %v) error: %v", pvc.ID, cErr)
				return cErr
			}
		}

		// create notebook on k8s
		log.Infof("creating k8s notebook(namespace: %v, name: %v)", k8sNotebook.Namespace, k8sNotebook.Name)
		cErr := m.k8sControllerClient.Create(context.Background(), k8sNotebook)
		if cErr != nil {
			log.Errorf("create k8s notebook(namespace: %v, name: %v) error: %v", k8sNotebook.Namespace, k8sNotebook.Name, cErr)
			return k8sErrToUI(cErr)
		}

		// create notebook in db
		notebook.LastActive = int(time.Now().Unix())
		//setVolumeMountID(notebook)
		log.Infof("creating db notebook(ID: %v)", notebook.ID)
		retNotebook, cErr = m.notebookStore.CreateNotebook(tx, notebook)
		if cErr != nil {
			log.Errorf("create db notebook(ID: %v) error: %v", notebook.ID, err)
			return cErr
		}
		return nil
	})
	if err != nil {
		if newK8sPvc != nil {
			_ = m.k8sControllerClient.Delete(context.Background(), newK8sPvc)
		}
		_ = m.k8sControllerClient.Delete(context.Background(), k8sNotebook)
		return nil, err
	}

	// async report data to pulsar
	data_report.AsyncDataReportMsg(notebook, m.conf, entity.NbActionCreate)
	return retNotebook, nil
}

// 更新notebook
func (m *Manager) ModifyNotebook(id string, modifyNotebook *entity.ModifyNotebook) (*entity.Notebook, error) {
	notebook, err := m.notebookStore.GetNotebook(id)
	if err != nil {
		return nil, entity.NewErrorf(entity.CodeParamInvalid, entity.MsgNotebookNotExisted, id)
	}

	// TODO check if have enough resource

	// create notebook in db
	log.Infof("updating db notebook(ID: %v)", notebook.ID)
	err = m.notebookStore.UpdateNotebook(id, modifyNotebook)
	if err != nil {
		log.Errorf("updating db notebook(ID: %v) error: %v", notebook.ID, err)
		return nil, err
	}
	notebook.ModifyNotebook = *modifyNotebook
	return notebook, nil
}

// 获取notebook日志
func (m *Manager) GetNotebookLogs(id string) (notebookLog string, err error) {
	notebook, err := m.notebookStore.GetNotebook(id)
	if err != nil {
		return "", entity.NewErrorf(entity.CodeParamInvalid, entity.MsgNotebookNotExisted, id)
	}
	count := int64(LogsCount)
	opts := &corev1.PodLogOptions{
		Follow:    false,
		Container: getK8sNotebookName(notebook),
		TailLines: &count,
	}
	request := m.k8sClient.CoreV1().Pods(notebook.GetNamespace()).GetLogs(getPodName(notebook), opts)
	readCloser, err := request.Stream()
	if err != nil {
		log.Errorf("error: %v", err)
		return "", entity.NewErrorf(entity.NotebookErrorModule, entity.MsgReadPodLogFailed)
	}
	defer readCloser.Close()
	readLogs := bufio.NewReader(readCloser)
	var logBuffer bytes.Buffer
	for {
		logBytes, err := readLogs.ReadBytes('\n')
		logBuffer.WriteString(string(logBytes))
		if err != nil {
			if err != io.EOF {
				break
			}
			return logBuffer.String(), nil
		}
	}
	return logBuffer.String(), nil
}

// 获取notebook列表
func (m *Manager) ListNotebook(option *entity.ProjectListOption) (total int, list []*entity.Notebook, err error) {
	total, list, _ = m.notebookStore.ListNotebook(option)
	// get status
	for _, notebook := range list {
		k8sNotebook := &v1.Notebook{}
		objectKey := client.ObjectKey{
			Namespace: notebook.GetNamespace(),
			Name:      getK8sNotebookName(notebook),
		}
		err = m.k8sControllerClient.Get(context.Background(), objectKey, k8sNotebook)
		if err != nil {
			if !errors.IsNotFound(err) {
				log.Errorf("get k8s notebook(ID: %v) error: %v", notebook.ID, err)
				return 0, nil, err
			}
			k8sNotebook = nil
		}
		notebook.Status = getStatus(k8sNotebook)
		notebook.GpuName = getGpuEnumName(notebook.GpuName)
		if notebook.Status == entity.NotebookRunning {
			url := getNotebookUrl(k8sNotebook)
			notebook.Url = url
			notebook.VscodeUrl = getVsCodeUrl(url)
		}
		if notebook.Status != entity.NotebookStopped {
			notebook.ClearLeftTime = notebook.LastActive + StopInactiveExpireTime - int(time.Now().Unix())
		}
	}
	return total, list, nil
}

func getGpuEnumName(value string) string {
	m := map[string]string{
		"genius_vgpu_8G":  "GeniusGpu8G",
		"genius_vgpu_16G": "GeniusGpu16G",
		"genius_vgpu_24G": "GeniusGpu24G",
		"genius_vgpu_32G": "GeniusGpu32G",
		"genius_vgpu_48G": "GeniusGpu48G",
		"genius_vgpu_64G": "GeniusGpu64G",
		"t4":              "T4",
		"a30":             "A30",
		"v100":            "V100",
		"a100":            "A100",
		"a800":            "A800",
		"rtx4090":         "RTX4090",
	}
	return m[value]
}

func getVsCodeUrl(url string) string {
	vscodeUrl := ""
	if url != "" {
		if strings.HasSuffix(url, "/") {
			vscodeUrl = url + "theia/"
		} else {
			vscodeUrl = url + "/theia/"
		}
	} else {
		log.Warnf("url is empty")
	}
	log.Infof("vscode url: %v", vscodeUrl)
	return vscodeUrl
}

// 删除notebook
func (m *Manager) DeleteNotebook(id string) error {
	notebook, err := m.notebookStore.GetNotebook(id)
	if err != nil {
		return entity.NewErrorf(entity.CodeParamInvalid, entity.MsgNotebookNotExisted, id)
	}

	k8sNotebook := &v1.Notebook{}
	objectKey := client.ObjectKey{
		Namespace: notebook.GetNamespace(),
		Name:      getK8sNotebookName(notebook),
	}
	err = m.k8sControllerClient.Get(context.Background(), objectKey, k8sNotebook)
	if err != nil {
		if !errors.IsNotFound(err) {
			log.Errorf("get k8s notebook(ID: %v) error: %v", notebook.ID, err)
			return err
		}
	} else {
		status := getStatus(k8sNotebook)
		if status == entity.NotebookStarting || status == entity.NotebookRunning {
			return entity.NewErrorf(entity.CodeParamInvalid, entity.MsgNotebookIsRunningCannotDelete, notebook.Name)
		}
		// delete notebook on k8s
		log.Infof("deleting k8s notebook(namespace: %v, name: %v)", notebook.GetNamespace(), notebook.Name)
		err = m.k8sControllerClient.Delete(context.Background(), k8sNotebook)
		if err != nil && !errors.IsNotFound(err) {
			log.Infof("delete notebook(namespace: %v, name: %v) on k8s error: %v",
				notebook.GetNamespace(), notebook.Name, err)
			return entity.NewCodeError(entity.CodeSystemError)
		}
	}

	// delete notebook in db
	log.Infof("deleting db notebook(ID: %v)", notebook.ID)
	if err = m.notebookStore.DeleteNotebook(notebook.ID); err != nil {
		log.Errorf("delete notebook(namespace: %v, name: %v) in db error: %v",
			notebook.GetNamespace(), notebook.Name, err)
		return entity.NewCodeError(entity.CodeSystemError)
	}

	// async report data to pulsar
	data_report.AsyncDataReportMsg(notebook, m.conf, entity.NbActionDelete)

	return nil
}

// 删除user notebook records
func (m *Manager) DeleteUserRecords(userId string) error {
	return m.db.Transaction(func(tx *gorm.DB) error {
		err := m.notebookStore.DeleteNotebookRecords(tx, userId)
		if err != nil {
			return err
		}
		err = m.pvcStore.DeletePvcRecords(tx, userId)
		if err != nil {
			return err
		}
		return nil
	})
}

// 删除project notebooks
func (m *Manager) DeleteNotebookByProjectId(projectId string) error {
	return m.db.Transaction(func(tx *gorm.DB) error {
		list, _ := m.notebookStore.GetNotebookByProjectId(tx, projectId)
		for _, notebook := range list {
			k8sNotebook := &v1.Notebook{}
			objectKey := client.ObjectKey{
				Namespace: notebook.GetNamespace(),
				Name:      getK8sNotebookName(notebook),
			}
			err := m.k8sControllerClient.Get(context.Background(), objectKey, k8sNotebook)
			if err != nil {
				if !errors.IsNotFound(err) {
					log.Errorf("get k8s notebook(ID: %v) error: %v", notebook.ID, err)
					return err
				}
				k8sNotebook = nil
			}
			notebook.Status = getStatus(k8sNotebook)
			id := notebook.ID
			if notebook.Status == entity.NotebookStopped {
				err := m.DeleteNotebook(id)
				if err != nil {
					return err
				}
			}
			if notebook.Status != entity.NotebookStopped {
				err := m.StopNotebook(id)
				if err != nil {
					return err
				}
				err1 := m.DeleteNotebook(id)
				if err1 != nil {
					return err1
				}
			}
		}
		return nil
	})
}

// 启动notebook
func (m *Manager) StartNotebook(id string) error {
	notebook, err := m.notebookStore.GetNotebook(id)
	if err != nil {
		return entity.NewErrorf(entity.CodeParamInvalid, entity.MsgNotebookNotExisted, id)
	}

	err = m.notebookStore.UpdateNotebookActive(id)
	if err != nil {
		log.Errorf("update notebook error: %v", err)
	}

	k8sNotebook := notebook2K8sNotebook(notebook, m.conf)
	// create notebook on k8s
	log.Infof("creating k8s notebook(namespace: %v, name: %v)", notebook.GetNamespace(), notebook.Name)
	err = m.k8sControllerClient.Create(context.Background(), k8sNotebook)
	if err != nil && !errors.IsAlreadyExists(err) {
		log.Errorf("create notebook(namespace: %v, name: %v) on k8s error: %v",
			notebook.GetNamespace(), notebook.Name, err)
		return k8sErrToUI(err)
	}
	// async report data to pulsar
	data_report.AsyncDataReportMsg(notebook, m.conf, entity.NbActionStart)
	return nil
}

// 停止notebook
func (m *Manager) StopNotebook(id string) error {
	notebook, err := m.notebookStore.GetNotebook(id)
	if err != nil {
		return entity.NewErrorf(entity.CodeParamInvalid, entity.MsgNotebookNotExisted, id)
	}

	// delete notebook on k8s
	k8sNotebook := &v1.Notebook{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: notebook.GetNamespace(),
			Name:      getK8sNotebookName(notebook),
		},
	}
	log.Infof("deleting k8s notebook(namespace: %v, name: %v)", notebook.GetNamespace(), notebook.Name)
	err = m.k8sControllerClient.Delete(context.Background(), k8sNotebook)
	if err != nil && !errors.IsNotFound(err) {
		log.Errorf("delete notebook(namespace: %v, name: %v) on k8s error: %v",
			notebook.GetNamespace(), notebook.Name, err)
		return entity.NewCodeError(entity.CodeSystemError)
	}
	// async report data to pulsar
	data_report.AsyncDataReportMsg(notebook, m.conf, entity.NbActionStop)
	return nil
}

// 更新notebook的活跃时间
func (m *Manager) ActiveNotebook(id string) error {
	err := m.notebookStore.UpdateNotebookActive(id)
	if err != nil {
		log.Errorf("update notebook error: %v", err)
		return err
	}
	return err
}

// 停止不活跃的notebook
func (m *Manager) StopInactiveNotebook(expireTime int64) error {
	expireTimestamp := time.Now().Unix() - expireTime
	notebooks, err := m.notebookStore.GetInactiveExpire(expireTimestamp)
	if err != nil {
		log.Errorf("get notebooks error: %v", err)
		return err
	}
	for _, notebook := range notebooks {
		// delete notebook on k8s
		k8sNotebook := &v1.Notebook{
			ObjectMeta: metav1.ObjectMeta{
				Namespace: notebook.GetNamespace(),
				Name:      getK8sNotebookName(notebook),
			},
		}
		err = m.k8sControllerClient.Delete(context.Background(), k8sNotebook)
		if err != nil {
			if !errors.IsNotFound(err) {
				log.Errorf("delete notebook(namespace: %v, name: %v) on k8s error: %v",
					notebook.GetNamespace(), notebook.Name, err)
			}
		} else {
			log.Infof("deleted k8s notebook(namespace: %v, name: %v)", notebook.GetNamespace(), notebook.Name)
		}
		// async report data to pulsar
		data_report.AsyncDataReportMsg(notebook, m.conf, entity.NbActionTimeoutStop)
	}
	return nil
}

// 获取需要新建的notebook
func getNewPvcByNotebook(notebook *entity.Notebook) *corev1.PersistentVolumeClaim {
	if !notebook.NewVolume {
		return nil
	}

	size := resource.MustParse(fmt.Sprintf("%dGi", notebook.WorkspaceSize))
	newPvc := &corev1.PersistentVolumeClaim{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: notebook.GetNamespace(),
			Name:      notebook.VolumeName,
		},
		Spec: corev1.PersistentVolumeClaimSpec{
			AccessModes: []corev1.PersistentVolumeAccessMode{
				corev1.ReadWriteMany,
			},
			Resources: corev1.ResourceRequirements{
				Requests: corev1.ResourceList{
					corev1.ResourceStorage: size,
				},
			},
		},
	}
	return newPvc
}

// notebook2K8sNotebook notebook结构体转成k8s结构体
func notebook2K8sNotebook(notebook *entity.Notebook, conf *config.Conf) *v1.Notebook {
	n := notebook2K8sNotebookTemplate(notebook)
	// vGpu
	if notebook.IsVGpu {
		n.Spec.Template.Spec.SchedulerName = schedulerName
		n.Spec.Template.Spec, _ = handleVGpuEnv(notebook, n.Spec.Template.Spec)
		// Gpu
	} else if !notebook.IsVGpu {
		n.Spec.Template.Spec.NodeSelector = map[string]string{
			GpuNodeSelectorLabelKey: notebook.GpuName,
		}
	}
	// cpu
	if notebook.GpuNum == 0 {
		n.Spec.Template.Spec.NodeSelector = map[string]string{
			NodeSelectorLabelKey: NodeSelectorLabelValue,
		}
	}
	n.Spec.Template.Spec, _ = getModelTrainWorkspace(notebook.UserID, n.Spec.Template.Spec, conf)
	return n
}

func handleVGpuEnv(notebook *entity.Notebook, podSpec corev1.PodSpec) (corev1.PodSpec, interface{}) {

	vGpuNum := fmt.Sprintf("%d", notebook.GpuNum)
	groupId := fmt.Sprintf("%s-%s", notebook.Name, notebook.ProjectID)
	podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, corev1.EnvVar{
		Name:  "ORION_VGPU",
		Value: vGpuNum,
	})
	podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, corev1.EnvVar{
		Name:  "ORION_DEVICE_NAME",
		Value: notebook.GpuName,
	})
	podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, corev1.EnvVar{
		Name:  "ORION_RESERVED",
		Value: "0",
	})
	podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, corev1.EnvVar{
		Name:  "ORION_CROSS_NODE",
		Value: "1",
	})
	podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, corev1.EnvVar{
		Name:  "ORION_CLIENT_ID",
		Value: notebook.UserID,
	})
	podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, corev1.EnvVar{
		Name:  "ORION_GROUP_ID",
		Value: groupId,
	})
	podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, corev1.EnvVar{
		Name:  "ORION_TASK_IDLE_TIME",
		Value: idleTime,
	})
	podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, corev1.EnvVar{
		Name:  "ORION_SCHEDULE_RULE",
		Value: schedulerRule,
	})

	return podSpec, nil
}

func getModelTrainWorkspace(userId string, podSpec corev1.PodSpec, conf *config.Conf) (corev1.PodSpec, error) {
	//volumes := podSpec.Volumes
	client := &http.Client{
		Timeout: time.Second * 30,
	}

	// 创建一个GET请求
	req, err := http.NewRequest("GET", "http://"+conf.TrainManger.Url+":"+conf.TrainManger.Port+GetTrainModelWorkspaceUrl, nil)
	if err != nil {
		fmt.Println("getModelTrainWorkspace创建请求失败:", err)
		return podSpec, err
	}

	// 添加Header
	req.Header.Add("kubeflow-userid", userId)

	// 发起请求
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("getModelTrainWorkspace发送请求失败:", err)
		return podSpec, err
	}

	// 处理响应
	fmt.Println("getModelTrainWorkspace响应状态码:", resp.StatusCode)
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	// 有错误则返回
	if err != nil {
		fmt.Println(err.Error())
		return podSpec, err
	}

	var resResult WorkspaceRes
	err = json.Unmarshal([]byte(string(body)), &resResult)
	if err != nil {
		fmt.Println(err.Error())
		return podSpec, err
	}

	fmt.Println("getModelTrainWorkspace创建请求结果:", string(body))

	for _, workspace := range resResult.Data.Workspaces {
		volumeName := uuid.New().String()
		pathType := corev1.HostPathDirectoryOrCreate
		podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{
			Name: volumeName,
			VolumeSource: corev1.VolumeSource{
				HostPath: &corev1.HostPathVolumeSource{
					Type: &pathType,
					Path: workspace,
				},
			},
		})

		podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, corev1.VolumeMount{
			Name:      volumeName,
			MountPath: workspace,
		})

	}

	return podSpec, nil
}

func notebook2K8sNotebookTemplate(notebook *entity.Notebook) *v1.Notebook {
	cpu := resource.MustParse(fmt.Sprintf("%dm", notebook.CpuNum*cpuUnit))
	vgpu := resource.MustParse(fmt.Sprintf("%d", notebook.GpuNum))
	nvidiaGpu := resource.MustParse(fmt.Sprintf("%d", notebook.GpuNum))
	memory := resource.MustParse(fmt.Sprintf("%dMi", notebook.MemorySize*memoryUnit))
	mountMode := corev1.MountPropagationHostToContainer
	pathType := corev1.HostPathDirectoryOrCreate

	if notebook.IsVGpu {
		nvidiaGpu = resource.MustParse(fmt.Sprintf("%d", 0))
	} else {
		vgpu = resource.MustParse(fmt.Sprintf("%d", 0))
	}

	ret := &v1.Notebook{
		ObjectMeta: metav1.ObjectMeta{
			Name:      getK8sNotebookName(notebook),
			Namespace: notebook.GetNamespace(),
			Annotations: map[string]string{
				"kubernetes.io/ingress-bandwidth": "10M",
				"kubernetes.io/egress-bandwidth":  "10M",
			},
		},
		Spec: v1.NotebookSpec{
			Template: v1.NotebookTemplateSpec{
				Spec: corev1.PodSpec{
					Volumes: append([]corev1.Volume{
						{
							Name: notebook.VolumeName,
							VolumeSource: corev1.VolumeSource{
								PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
									ClaimName: notebook.VolumeName,
								},
							},
						},
						{
							Name: MemoryShmName,
							VolumeSource: corev1.VolumeSource{
								EmptyDir: &corev1.EmptyDirVolumeSource{
									Medium: corev1.StorageMediumMemory,
								},
							},
						},
						{
							Name: "mntdatasets",
							VolumeSource: corev1.VolumeSource{
								HostPath: &corev1.HostPathVolumeSource{
									Path: fmt.Sprintf("/mnt/datasets/%s/%s", notebook.GetNamespace(), notebook.Name),
									Type: &pathType,
								},
							},
						},
					}, lxcfs.LxcfsVolume...),
					Containers: []corev1.Container{
						{
							Name:  getK8sNotebookName(notebook),
							Image: notebook.Image,
							Resources: corev1.ResourceRequirements{
								Requests: corev1.ResourceList{
									corev1.ResourceCPU:    cpu,
									corev1.ResourceMemory: memory,
								},
								Limits: corev1.ResourceList{
									corev1.ResourceCPU:    cpu,
									corev1.ResourceMemory: memory,
								},
							},
							ReadinessProbe: &corev1.Probe{
								Handler: corev1.Handler{
									Exec: &corev1.ExecAction{
										Command: []string{"/bin/sh", "-c", "nc -vz localhost 8888"},
									},
								},
								InitialDelaySeconds: 1,
								TimeoutSeconds:      2,
								PeriodSeconds:       60,
								SuccessThreshold:    1,
								FailureThreshold:    15,
							},
							VolumeMounts: append([]corev1.VolumeMount{
								{
									Name:      MemoryShmName,
									MountPath: MemoryShmMouthPath,
								},
								{
									Name:      notebook.VolumeName,
									MountPath: entity.NotebookHomePath,
								},
								{
									Name:      notebook.VolumeName,
									MountPath: entity.ModelsPath,
									SubPath:   entity.VolumeModelsSubPath,
								},
								{
									Name:             "mntdatasets",
									MountPath:        "/mnt/datasets",
									MountPropagation: &mountMode,
								},
							}, lxcfs.LxcfsVolumeMount...),
							Env: []corev1.EnvVar{
								{
									Name:  "AWS_ENDPOINT_URL",
									Value: "http://minio-service.kubeflow:9000",
								},
								{
									Name:  "PROJECT_ID",
									Value: notebook.ProjectID,
								},
							},
						}},
					DNSConfig: &corev1.PodDNSConfig{
						Nameservers: []string{
							"200.200.10.199",
						},
						Searches: []string{
							"default.svc.cluster.local",
							"svc.cluster.local",
							"cluster.local",
						},
						Options: []corev1.PodDNSConfigOption{},
					},
				},
			},
		},
	}

	if notebook.IsVGpu {
		ret.Spec.Template.Spec.Containers[0].Resources.Requests[v1ResourceGpu] = vgpu
		ret.Spec.Template.Spec.Containers[0].Resources.Limits[v1ResourceGpu] = vgpu
	} else if notebook.GpuNum > 0 {
		ret.Spec.Template.Spec.Containers[0].Resources.Requests[nvidiaResourceGpu] = nvidiaGpu
		ret.Spec.Template.Spec.Containers[0].Resources.Limits[nvidiaResourceGpu] = nvidiaGpu
		ret.Spec.Template.Spec.Containers[0].Env = append(ret.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{
			Name:  "ORION_DEVICE_ENABLE",
			Value: "0",
		})
	}

	return ret
}

func (m *Manager) CheckNotebookNameIfExist(notebookName string, userID string) bool {
	option := &entity.ProjectListOption{}
	option.UserID = userID
	var names []string
	_, notebooks, err := m.notebookStore.ListNotebook(option)
	if err != nil {
		return false
	}
	for _, notebook := range notebooks {
		names = append(names, notebook.Name)
	}
	set := make(map[string]struct{})
	for _, value := range names {
		set[value] = struct{}{}
	}
	if _, ok := set[notebookName]; !ok {
		return false
	}
	return true
}

func getK8sNotebookName(notebook *entity.Notebook) string {
	return notebook.Name + PodNameSeparator + notebook.ProjectID[len(notebook.ProjectID)-5:]
}

func getPodName(notebook *entity.Notebook) string {
	return getK8sNotebookName(notebook) + StatefulSetPodId
}

func genIDByProjectUidName(userID, name string, projectID string) string {
	return userID + ResourceIDSeparator + name + ResourceIDSeparator + projectID
}

func genIDByUidName(userID, name string) string {
	return userID + ResourceIDSeparator + name
}

//func setVolumeMountID(notebook *entity.Notebook) {
//	for i, _ := range notebook.VolumeMounts {
//		notebook.VolumeMounts[i].ID = genIDByUidName(notebook.UserID, notebook.VolumeMounts[i].Name)
//	}
//}

// 根据k8s的notebook获取状态
func getStatus(k8sNotebook *v1.Notebook) entity.NotebookStatus {
	if k8sNotebook == nil {
		return entity.NotebookStopped
	}
	if k8sNotebook.Status.ReadyReplicas > 0 {
		return entity.NotebookRunning
	}
	if k8sNotebook.Status.ReadyReplicas == 0 &&
		k8sNotebook.GetCreationTimestamp().Add(time.Second*NotebookFailedTimeLimit).Before(time.Now()) {
		return entity.NotebookFailed
	}
	return entity.NotebookStarting
}

// 获取notebook的url
func getNotebookUrl(k8sNotebook *v1.Notebook) string {
	return fmt.Sprintf("/notebook/%s/%s/", k8sNotebook.Namespace, k8sNotebook.Name)
}

type WorkspaceRes struct {
	Code    string `json:"code"`
	Msg     string `json:"msg"`
	Success bool   `json:"success"`
	Data    struct {
		Workspaces []string `json:"workspaces"`
	} `json:"data"`
}
